package org.example.crawler;

import cn.hutool.core.util.RandomUtil;
import cn.hutool.json.JSONUtil;
import lombok.AllArgsConstructor;
import org.example.crawler.domain.Flaw;
import org.example.utils.HttpUtils;
import org.example.utils.RequestUtils;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;

/**
 * @Author lyz
 * @Date 2025/3/27 14:25
 */
@AllArgsConstructor
public class PageProcesser {

    private RequestParam requestParam;


    //处理
    public void process(RequestParam requestParam) throws Exception {
        AtomicInteger dataNum = new AtomicInteger();
        List<Flaw> dataList = new ArrayList<>();
        int max = requestParam.getMax();
        HttpUtils.configTimeout(requestParam.getTimeOut(), requestParam.getTimeOut());
        Map<String, String> headers = RequestUtils.getHeader();
        for (int i = 0; i < requestParam.getTotalPage(); i++) {
            String uri = requestParam.getUri();
            int offset = i * requestParam.getOffset();
            uri = String.format(uri, max, offset);
            //获取分页页面
            String pageHtml = RequestUtils.sendRequest(uri, headers);
            //解析分页数据
            Document document = Jsoup.parse(pageHtml);
            Elements tbodyElements = document.getElementsByTag("tbody");
            if (tbodyElements.size() > 0) {
                Element tbody = tbodyElements.get(0);
                Elements elements = tbody.getElementsByTag("a");
                List<String> detailsHref = new ArrayList<>();
                elements.stream().forEach(element -> {
                    detailsHref.add(element.attr("href"));
                });
                for (String detailHref : detailsHref) {
                    //请求详情
                    String html = RequestUtils.sendRequest(requestParam.getPrefixUri() + detailHref, headers);
                    //计算拉取数量
                    dataNum.incrementAndGet();
                    //解析具体数据
                    document = Jsoup.parse(html);
                    Elements tables = document.getElementsByClass("gg_detail");
                    if (tables.size() > 0) {
                        Element table = tables.get(0);
                        Elements trs = table.getElementsByTag("tr");
                        Flaw flaw = new Flaw();

                        for (Element tr : trs) {
                            Elements tds = tr.getElementsByTag("td");
                            if (tds.size() == 2) {
                                flaw.setField(tds.get(0).text(), tds.get(1).text());
                            }
                        }

                        dataList.add(flaw);
                    }
                    //降低频率
                    int time = RandomUtil.randomInt(0, 9) * 1000;
                    Thread.sleep(time);
                }
            }
        }

        System.out.println("获取的数据：" + JSONUtil.toJsonStr(dataList));
    }

    public static void main(String[] args) throws Exception {
        RequestParam param = new RequestParam();
        param.setTimeOut(30000);
        param.setTotalPage(1);
        param.setMax(10);
        param.setOffset(10);
        param.setPrefixUri("https://www.cnvd.org.cn");
        param.setUri("https://www.cnvd.org.cn/flaw/typeResult?typeId=27&max=%s&offset=%s");
        PageProcesser pageProcesser = new PageProcesser(param);
        pageProcesser.process(param);
    }


}
